# coding=utf8
import time
import re
import requests
import json
import pandas as pd
import numpy as np
import math
from shapely.geometry import Polygon, Point
import os
from copy import deepcopy

homestay = pd.read_csv("/home/mm/Documents/bnb_data_wrangler/homestay.csv", dtype={'town_code': int})


def jacaard_similarity(y_true, y_pred):
    return len(y_pred & y_true) / len(y_pred and y_true)


def drop_dup_house_info(df_group):
    len_df_group = len(df_group.index)
    if len_df_group == 1:
        return df_group
    else:
        similar_pairs = []
        df_group_values = df_group[['hyper_link', 'house_info_set']].values
        scores_dict = {x: 0 for x in df_group_values['hyper_link']}
        for i in range(0, len_df_group - 1):
            for j in range(i + 1, len_df_group):
                similarity = jacaard_similarity(df_group_values[i]['house_info_set'], df_group_values[j]['house_info_set'])
                scores_dict[df_group_values[i]['hyper_link']] += similarity
                scores_dict[df_group_values[j]['hyper_link']] += similarity
                if similarity > 0.8:
                    similar_pairs.append([df_group_values[i]['hyper_link'], df_group_values[j]['hyper_link']])
        score_sorted = sorted([{hyperlink: scores_dict[hyperlink]} for hyperlink in scores_dict.keys()], key=lambda x: scores_dict[x])
        high_socres = []
        for pairs in similar_pairs:
            high_socres += pairs
        set_high_scores = set(high_socres)
        set_high_scores_copy = deepcopy(set_high_scores)
        for pairs in similar_pairs:
            link1, link2 = pairs
            if len(set_high_scores) == 1:
                break
            elif score_sorted.index(link1) > score_sorted.index(link2) and link1 in set_high_scores:
                set_high_scores.remove(link1)
            elif score_sorted.index(link1) < score_sorted.index(link2) and link2 in set_high_scores:
                set_high_scores.remove(link2)
        drop_links = list(set_high_scores_copy - set_high_scores)
        result = df_group.drop(df_group.isin({'hyper_link': drop_links}).index)
        return result
